# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html


# useful for handling different item types with a single interface
from itemadapter import ItemAdapter

import pandas as pd

# 数据存储在本地json文件中
class ScrapyDytt835NewPipeline:

    def open_spider(self,spider):
        self.fp = open('mv.json','w',encoding='utf-8')

    def process_item(self, item, spider):
        self.fp.write(str(item))
        return item

    def close_spider(self,spider):
        self.fp.close()

# 使用pandas将数据存储在excel中
class dytt8newDownloadPipeline:

    def __init__(self):
        # 创建一个空的数据框
        self.df = pd.DataFrame(columns=['name','src','href'])

    def process_item(self, item, spider):
        # 将数据添加到数据框中
        # item['name'] = item.get('name','')这样的写法是因为如果返回的item中的一些字段值为None时，pandas不会将
        # None类型的值添加到DataFrame中，会导致程序错误，也可以不用
        item['name'] = item.get('name','')
        item['src'] = item.get('src', '')
        item['href'] = item.get('href', '')
        # 将item字典对象转为series对象，再添加到DataFrame中
        series = pd.Series(item)
        self.df = self.df.append(series,ignore_index=True)

        return item

    def close_spider(self,spider):
        self.df.to_excel('result.xlsx',index=False)